{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from nltk.corpus import stopwords\n",
    "import numpy as np\n",
    "stop_words = stopwords.words('english')\n",
    "import warnings\n",
    "warnings.filterwarnings(action='ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "path = \"../data/\"\n",
    "\n",
    "train = pd.read_pickle(path + \"train_clean.pkl\")\n",
    "valid = pd.read_pickle(path + \"valid_clean.pkl\")\n",
    "dev = pd.read_pickle(path+'dev_clean.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data_all = pd.concat([train,valid,dev])\n",
    "data_all.reset_index(inplace=1,drop=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def generate_stats_len_feature(x,mode='max'):\n",
    "    wl = str(x).strip().split()\n",
    "    wlen = []\n",
    "    for w in wl:\n",
    "        wlen.append(len(w))\n",
    "    if mode=='max':\n",
    "        return np.max(wlen)\n",
    "    elif mode=='mean':\n",
    "        return np.mean(wlen)\n",
    "    else:\n",
    "        return np.min(wlen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "res = pd.DataFrame()\n",
    "res['len_sen'] = data_all.context.apply(lambda x: len(str(x)))\n",
    "res['len_word'] = data_all.context.apply(lambda x: len(str(x).strip().split()))\n",
    "res['len_max_word'] = data_all.context.apply(lambda x: generate_stats_len_feature(x, 'max'))\n",
    "res['len_min_word'] = data_all.context.apply(lambda x: generate_stats_len_feature(x, 'min'))\n",
    "res['len_mean_word'] = data_all.context.apply(lambda x: generate_stats_len_feature(x, 'mean'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>len_sen</th>\n",
       "      <th>len_word</th>\n",
       "      <th>len_max_word</th>\n",
       "      <th>len_min_word</th>\n",
       "      <th>len_mean_word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>640</td>\n",
       "      <td>117</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>4.478632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>831</td>\n",
       "      <td>150</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>4.546667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>84</td>\n",
       "      <td>7</td>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>5.714286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1065</td>\n",
       "      <td>217</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>3.912442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>261</td>\n",
       "      <td>55</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>3.763636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>367</td>\n",
       "      <td>76</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>3.842105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1446</td>\n",
       "      <td>265</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>4.460377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2089</td>\n",
       "      <td>366</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>4.710383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>689</td>\n",
       "      <td>122</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>4.655738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1271</td>\n",
       "      <td>252</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>4.043651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>402</td>\n",
       "      <td>83</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>3.855422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>3102</td>\n",
       "      <td>580</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>4.348276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>863</td>\n",
       "      <td>148</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>4.837838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2683</td>\n",
       "      <td>516</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>4.201550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>637</td>\n",
       "      <td>123</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>4.186992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>136</td>\n",
       "      <td>22</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2019</td>\n",
       "      <td>403</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>4.009926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1370</td>\n",
       "      <td>275</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>3.985455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1553</td>\n",
       "      <td>292</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>4.318493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1108</td>\n",
       "      <td>229</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>3.838428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>5281</td>\n",
       "      <td>981</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>4.384302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>633</td>\n",
       "      <td>131</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>3.839695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2501</td>\n",
       "      <td>488</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>4.125000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>213</td>\n",
       "      <td>44</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1.409091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>796</td>\n",
       "      <td>152</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>4.236842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>5513</td>\n",
       "      <td>1024</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>4.384766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>187</td>\n",
       "      <td>32</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>159</td>\n",
       "      <td>26</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1.615385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1132</td>\n",
       "      <td>211</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>4.369668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1496</td>\n",
       "      <td>275</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>4.443636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30890</th>\n",
       "      <td>2189</td>\n",
       "      <td>416</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>4.264423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30891</th>\n",
       "      <td>1061</td>\n",
       "      <td>204</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4.205882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30892</th>\n",
       "      <td>5691</td>\n",
       "      <td>1000</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>4.692000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30893</th>\n",
       "      <td>636</td>\n",
       "      <td>126</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>4.055556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30894</th>\n",
       "      <td>659</td>\n",
       "      <td>131</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>4.038168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30895</th>\n",
       "      <td>3860</td>\n",
       "      <td>717</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>4.383543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30896</th>\n",
       "      <td>2374</td>\n",
       "      <td>464</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>4.118534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30897</th>\n",
       "      <td>5976</td>\n",
       "      <td>1099</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>4.438581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30898</th>\n",
       "      <td>1742</td>\n",
       "      <td>337</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>4.172107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30899</th>\n",
       "      <td>395</td>\n",
       "      <td>73</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>4.424658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30900</th>\n",
       "      <td>2005</td>\n",
       "      <td>372</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>4.392473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30901</th>\n",
       "      <td>791</td>\n",
       "      <td>153</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>4.169935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30902</th>\n",
       "      <td>223</td>\n",
       "      <td>42</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1.571429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30903</th>\n",
       "      <td>2640</td>\n",
       "      <td>453</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>4.830022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30904</th>\n",
       "      <td>3654</td>\n",
       "      <td>635</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>4.755906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30905</th>\n",
       "      <td>1903</td>\n",
       "      <td>385</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>3.942857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30906</th>\n",
       "      <td>676</td>\n",
       "      <td>137</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>3.941606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30907</th>\n",
       "      <td>4134</td>\n",
       "      <td>730</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>4.664384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30908</th>\n",
       "      <td>1517</td>\n",
       "      <td>282</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>4.382979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30909</th>\n",
       "      <td>2910</td>\n",
       "      <td>570</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>4.105263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30910</th>\n",
       "      <td>7503</td>\n",
       "      <td>1387</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>4.409517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30911</th>\n",
       "      <td>532</td>\n",
       "      <td>105</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>4.076190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30912</th>\n",
       "      <td>1055</td>\n",
       "      <td>188</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1.734043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30913</th>\n",
       "      <td>507</td>\n",
       "      <td>101</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>4.019802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30914</th>\n",
       "      <td>1008</td>\n",
       "      <td>178</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>4.668539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30915</th>\n",
       "      <td>966</td>\n",
       "      <td>173</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>4.589595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30916</th>\n",
       "      <td>967</td>\n",
       "      <td>177</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>4.463277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30917</th>\n",
       "      <td>706</td>\n",
       "      <td>140</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>4.050000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30918</th>\n",
       "      <td>2093</td>\n",
       "      <td>391</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>4.352941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30919</th>\n",
       "      <td>1536</td>\n",
       "      <td>300</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>4.123333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30920 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       len_sen  len_word  len_max_word  len_min_word  len_mean_word\n",
       "0          640       117            13             1       4.478632\n",
       "1          831       150            12             1       4.546667\n",
       "2           84         7            15             2       5.714286\n",
       "3         1065       217            12             1       3.912442\n",
       "4          261        55            11             1       3.763636\n",
       "5          367        76            10             1       3.842105\n",
       "6         1446       265            11             1       4.460377\n",
       "7         2089       366            15             1       4.710383\n",
       "8          689       122            13             1       4.655738\n",
       "9         1271       252            15             1       4.043651\n",
       "10         402        83            10             1       3.855422\n",
       "11        3102       580            13             1       4.348276\n",
       "12         863       148            14             1       4.837838\n",
       "13        2683       516            13             1       4.201550\n",
       "14         637       123            11             1       4.186992\n",
       "15         136        22             2             1       1.500000\n",
       "16        2019       403            11             1       4.009926\n",
       "17        1370       275            13             1       3.985455\n",
       "18        1553       292            13             1       4.318493\n",
       "19        1108       229            12             1       3.838428\n",
       "20        5281       981            12             1       4.384302\n",
       "21         633       131             9             1       3.839695\n",
       "22        2501       488            13             1       4.125000\n",
       "23         213        44             2             1       1.409091\n",
       "24         796       152            11             1       4.236842\n",
       "25        5513      1024            14             1       4.384766\n",
       "26         187        32             3             1       1.500000\n",
       "27         159        26             3             1       1.615385\n",
       "28        1132       211            12             1       4.369668\n",
       "29        1496       275            13             1       4.443636\n",
       "...        ...       ...           ...           ...            ...\n",
       "30890     2189       416            14             1       4.264423\n",
       "30891     1061       204            10             1       4.205882\n",
       "30892     5691      1000            14             1       4.692000\n",
       "30893      636       126            11             1       4.055556\n",
       "30894      659       131            11             1       4.038168\n",
       "30895     3860       717            12             1       4.383543\n",
       "30896     2374       464            12             1       4.118534\n",
       "30897     5976      1099            14             1       4.438581\n",
       "30898     1742       337            11             1       4.172107\n",
       "30899      395        73            11             1       4.424658\n",
       "30900     2005       372            13             1       4.392473\n",
       "30901      791       153            13             1       4.169935\n",
       "30902      223        42             2             1       1.571429\n",
       "30903     2640       453            17             1       4.830022\n",
       "30904     3654       635            15             1       4.755906\n",
       "30905     1903       385            13             1       3.942857\n",
       "30906      676       137            12             1       3.941606\n",
       "30907     4134       730            15             1       4.664384\n",
       "30908     1517       282            13             1       4.382979\n",
       "30909     2910       570            13             1       4.105263\n",
       "30910     7503      1387            13             1       4.409517\n",
       "30911      532       105            12             1       4.076190\n",
       "30912     1055       188             5             1       1.734043\n",
       "30913      507       101            12             1       4.019802\n",
       "30914     1008       178            15             1       4.668539\n",
       "30915      966       173            12             1       4.589595\n",
       "30916      967       177            13             1       4.463277\n",
       "30917      706       140            11             1       4.050000\n",
       "30918     2093       391            14             1       4.352941\n",
       "30919     1536       300            11             1       4.123333\n",
       "\n",
       "[30920 rows x 5 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
